#!/usr/bin/python
import sys
from data_structs import Tweet
#from data_structs import Text
#from test_api import crawl_api
from crawl_api import crawl_api
from crawl_url import crawl_url
from api_controller import API_Controller
from datetime import datetime


def crawl(filename,prefix,directory,limit=5000):

    if directory[len(directory)-1]!="/":
       directory = directory + "/"

    api_c = API_Controller(4900,3600)
    f = open(filename)
    line = f.readline()
    count = 0
    affix = 0
    of = open(directory+prefix+"_"+str(affix),"w")
	
    progress = 0

    while line!="":
        tweet = Tweet(line)
        text = crawl_api(tweet, api_c)
        if text.get_content()=="":
            print "Empty crawl_url\n"
            text = crawl_url(tweet)
        
        of.write("URL\t"+tweet.get_url()+"\n")
        of.write("Type\t"+text.get_type()+"\n")
        try:
            of.write("Content\t"+text.get_content()+"\n")
        except Exception as e:
            of.write("Content\t"+text.get_content().encode('utf-8')+"\n")
        
        of.write("\n\n")
        count = count +1
        if count >= limit:
            count = 0
            of.close()
            affix = affix + 1
            of = open(directory+prefix+"_"+str(affix),"w")
        
        line = f.readline()
	
	progress = progress + 1
	print "prgress done:"+str(progress)+"\n"

    of.close()
    f.close()

if len(sys.argv)<5:
    print "usage: filename direcotry prefix limit"
else:
    t1 = datetime.now()
    filename = sys.argv[1]
    directory = sys.argv[2]
    prefix = sys.argv[3]
    limit = int(sys.argv[4])
    crawl(filename,prefix,directory,limit)
    t2 = datetime.now()
    c =t2 -t1
    print "totoal seconds:"+str(c.seconds)+"\n"
    
